{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import argparse\n",
    "import logging\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "import xgboost\n",
    "import time\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备工作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    from sklearn.cross_validation import KFold\n",
    "    __USE_OLD_KOLD = True\n",
    "except Exception:\n",
    "    from sklearn.model_selection import KFold\n",
    "    __USE_OLD_KOLD = False\n",
    "\n",
    "\n",
    "def get_train_test(X, y, kfold=False, kfold_k=None, kfold_shuffle=False):\n",
    "    if kfold is False:\n",
    "        yield X, y, None, None\n",
    "    else:\n",
    "        if __USE_OLD_KOLD:\n",
    "            kf = KFold(X.shape[0], n_folds=kfold_k, shuffle=kfold_shuffle)\n",
    "            for train_index, test_index in kf:\n",
    "                yield X[train_index], y[train_index], X[test_index], y[test_index]\n",
    "        else:\n",
    "            kf = KFold(n_splits=kfold_k, shuffle=kfold_shuffle)\n",
    "            for train_index, test_index in kf.split(X):\n",
    "                yield X[train_index], y[train_index], X[test_index], y[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def kfold_validation(X, y, kfold_k, model, vis=False, verbose=False):\n",
    "    all_score = 0\n",
    "    for i, (X_train, y_train, X_test, y_test) in enumerate(get_train_test(X, y, True, kfold_k, True)):\n",
    "    #     print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)\n",
    "        clf = model\n",
    "        clf.fit(X_train, y_train)\n",
    "        score = clf.score(X_test, y_test)\n",
    "        if verbose:\n",
    "            print('Score: %d / %d: %f' % (i + 1, kfold_k, score))\n",
    "        all_score += score\n",
    "        if vis:\n",
    "            y_predicted = clf.predict(X_test)\n",
    "            fig, ax = plt.subplots()\n",
    "            ax.scatter(y_test, y_predicted)\n",
    "            ax.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'k--', lw=4)\n",
    "            ax.set_xlabel('Observed')\n",
    "            ax.set_ylabel('Predicted')\n",
    "            plt.show()\n",
    "    if verbose:\n",
    "        print('avg score: %f' % (all_score / kfold_k))\n",
    "    return all_score / kfold_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./data/sourceAB.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 只用A特征训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = df.filter(regex='f_a.*', axis=1).values\n",
    "y = np.log(df['hotness'].values)\n",
    "X = StandardScaler().fit_transform(X)\n",
    "y = (y - y.mean() / y.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.455335576369\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for i in range(100):\n",
    "    scores.append(kfold_validation(\n",
    "        X, y,\n",
    "        kfold_k=5,\n",
    "        model = xgboost.XGBRegressor(n_estimators=90, max_depth=3),\n",
    "       vis = False\n",
    "    ))\n",
    "print(sum(scores) / len(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 只用B特征训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.filter(regex='f_b.*', axis=1).values\n",
    "y = np.log(df['hotness'].values)\n",
    "X = StandardScaler().fit_transform(X)\n",
    "y = (y - y.mean() / y.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.623573388834\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for i in range(100):\n",
    "    scores.append(kfold_validation(\n",
    "        X, y,\n",
    "        kfold_k=5,\n",
    "        model = xgboost.XGBRegressor(n_estimators=90, max_depth=3),\n",
    "       vis = False\n",
    "    ))\n",
    "print(sum(scores) / len(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A、B特征直接组合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = df.filter(regex='f_.*', axis=1).values\n",
    "y = np.log(df['hotness'].values)\n",
    "X = StandardScaler().fit_transform(X)\n",
    "y = (y - y.mean() / y.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.626470806624\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for i in range(100):\n",
    "    scores.append(kfold_validation(\n",
    "        X, y,\n",
    "        kfold_k=5,\n",
    "        model = xgboost.XGBRegressor(n_estimators=90, max_depth=3, min_child_weight=8.0),\n",
    "#         model = Pipeline([(\"1\", StandardScaler()), (\"2\", Lasso(alpha=0.02))]),\n",
    "       vis = False\n",
    "    ))\n",
    "print(sum(scores) / len(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def feature_importance_xgboost(X, y, X_names, n_models, model_para_ref={}, use_seed=False, task='classification', kfold=False, kfold_k=0, kfold_shuffle=False, scorer=None, verbose=False):\n",
    "    '''\n",
    "    计算多次xgboost模型，给出所有特征的重要程度打分\n",
    "\n",
    "    输入参数说明：\n",
    "        y: 代表目标列的字符， 如target\n",
    "        df: DataFrame。除了y的列剩下的列都用做训练\n",
    "        n_models: 训练多少次\n",
    "        model_para_ref: 形如：\n",
    "                para_ref =    {\n",
    "                    'max_depth': [2, 3, 4],\n",
    "                    'learning_rate': [0.1, 0.5, 1.0, 1.5],\n",
    "                    'objective': ['rank:pairwise']\n",
    "                }\n",
    "                每一个参数都给出一些候选。会随机在这些里面选择n_models个参数训练\n",
    "        use_seed: 如果为True，每次训练还会强行加入一个随机参数seed。\n",
    "        task: classification表示使用XGBClassifier，否则使用XGBRegressor\n",
    "        kfold、kfold_k、kfold_shuffle、scorer: kfold=True时这几个参数才有效果：\n",
    "                kfold_k: 使用多少折\n",
    "                kfold_shuffle: kfold时是否使用shuffle打乱\n",
    "                scorer: 形如scorer(model, X, y_gt)的函数，输出一个数表示模型的打分。使用kfold必须传入该函数否则kfold无意义。\n",
    "        verbose: 是否输出详细信息\n",
    "\n",
    "    输出结果：\n",
    "        kfold=False时，输出result，代表每个feature的importance，从大到小排列。\n",
    "        kfold=True时，输出result，avg_score，avg_score代表模型的平均打分。\n",
    "\n",
    "    举例：\n",
    "    如只想尝试一组参数，在不同seed下的效果：\n",
    "        model_para_ref = {\n",
    "            'depth': [1],\n",
    "            'learning_rate': [0.1]\n",
    "        }\n",
    "        use_seed=True\n",
    "    '''\n",
    "\n",
    "    logger = logging.getLogger('feature_importance_xgboost')\n",
    "    feature_name = X_names\n",
    "    feature_importance = np.zeros(len(feature_name))\n",
    "    score = 0\n",
    "    score_n = 0\n",
    "\n",
    "    for i in range(n_models):\n",
    "        para = {}\n",
    "        for key in model_para_ref:\n",
    "            choice = model_para_ref[key][random.randint(0, len(model_para_ref[key]) - 1)]  # 在每个参数中都随机选取一个\n",
    "            para[key] = choice\n",
    "        if use_seed:\n",
    "            para['seed'] = random.randint(0, 10000)\n",
    "\n",
    "        if verbose:\n",
    "            logger.warning('model %d / %d' % (i + 1, n_models))\n",
    "            logger.warning('use para: %s' % str(para))\n",
    "\n",
    "        # 分离x_train, y_train\n",
    "        for X_train, y_train, X_test, y_test in get_train_test(X_df.values, y_df.values, kfold, kfold_k, kfold_shuffle):\n",
    "            if task == 'classification':\n",
    "                model = xgboost.XGBClassifier(**para)\n",
    "            else:\n",
    "                model = xgboost.XGBRegressor(**para)\n",
    "            model.fit(X_train, y_train)\n",
    "            feature_importance += model.feature_importances_\n",
    "            if kfold is True:\n",
    "                score_i = scorer(model, X_test, y_test)\n",
    "                score += score_i\n",
    "                score_n += 1\n",
    "                if verbose:\n",
    "                    logger.warning('model %d score %f' % (i + 1, float(score_i)))\n",
    "\n",
    "    if kfold is True:\n",
    "        feature_importance = feature_importance / score_n\n",
    "    else:\n",
    "        feature_importance = feature_importance / n_models\n",
    "\n",
    "    result = pd.DataFrame(feature_importance, index=feature_name, columns=['importance'])\n",
    "    result = result.sort_values('importance', ascending=False)\n",
    "\n",
    "    if kfold is True:\n",
    "        return result, float(score) / score_n\n",
    "    else:\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
